# Updated by Wei 2013/07/22 night at school
from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

print "program begins..."
print "Updated on 2013/07/27 afternoon by Wei at school."
print "The sole purpose of this program is to generate the missing postings needed for red curve into a basic training file format."

# The variable for TOPKLabel
TOPKLabel = "TOP10" 
print "This program will extract the",TOPKLabel,"for the (input)testing file."

# extract the TOPK results, here is setting the K
# K can be set to 10,100,1000,10000
K = 0
if TOPKLabel == "TOP10": 
    K = 10
elif TOPKLabel == "TOP100":
    K = 100
elif TOPKLabel == "TOP1000":
    K = 1000
elif TOPKLabel == "TOP10000":
    K = 10000

# This simple part of logic is to extract those missing results in order to satisfy the prof
queryIDList = []
inputTestingFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TOPK_sorted_by_queryID/Training_Set_2013_07_19_sorted_by_queryID_middle_DOT1M_testing.arff"
inputTestingFileHandler = open(inputTestingFileName,"r")
for i in range(0,26):
    inputTestingFileHandler.readline()

for currentLine in inputTestingFileHandler.readlines():
    currentLineElements = currentLine.strip().split(" ")
    currentQueryIDInIntFormat = int( currentLineElements[1] )
    if currentQueryIDInIntFormat not in queryIDList:
        queryIDList.append(currentQueryIDInIntFormat)

queryIDList.sort(cmp=None, key=None, reverse=False)
print "len(queryIDList):",len(queryIDList)
# print "queryIDList:",queryIDList
inputTestingFileHandler.close()

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TrainingSetForTOPKMissingResults20130719"
outputFileHandler = open(outputFileName,"w")

# write the headline
outputFileHandler.write("SelectedRankIndex queryID externalTrecID internalDocID term partialBM25ScoreComponentPart1_IDF partialBM25ScoreComponentPart2_TF partialBM25 length_of_inverted_index term_freq_in_doc doc_words overallBM25Score rank_in_result_list" + "\n")

originalRawResultFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/rawResultsHead10KANDSemanticsTOP2MResults"
originalRawResultFileHandler = open(originalRawResultFileName,"r")

queryAuxDict = {}
inputAuxFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/rawResultsHead10KANDSemanticsTOP2MResultsAccessAuxFile_20130720"
inputFileHandler = open(inputAuxFileName,"r")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    qidInStringFormat = lineElements[0]
    # num of query results which we don't need. (lineElements[1])
    beginningPositionForThisQuery = int(lineElements[2])
    endingPositionForThisQuery = int(lineElements[3])
    currentTuple = (beginningPositionForThisQuery,endingPositionForThisQuery)
    queryAuxDict[qidInStringFormat] = currentTuple
    
print "----->","len(queryAuxDict):",len(queryAuxDict)
print "queryAuxDict['1']:",queryAuxDict['1']
print "queryAuxDict['20']:",queryAuxDict['20']

for currentQIDInIntFormat in queryIDList:
    currentQIDInStrFormat = str(currentQIDInIntFormat)
    (beginningPositionInIntFormatInOriginalRawResultFile,endingPositionInIntFormatInOriginalRawResultFile) = queryAuxDict[currentQIDInStrFormat]
    if beginningPositionInIntFormatInOriginalRawResultFile != -1 and endingPositionInIntFormatInOriginalRawResultFile != -1:
        originalRawResultFileHandler.seek(beginningPositionInIntFormatInOriginalRawResultFile)
        
        for j in range(0,3):
            originalRawResultFileHandler.readline()
        
        # print "(right line for dict):"
        # key: term index
        # value: term
        currentQueryTermIndexDict = {}
        currentLine = originalRawResultFileHandler.readline()
        currentLineElements = currentLine.strip().split(" ")
        for element in currentLineElements:
            term = element.split(":")[0]
            termIndex = int(element.split(":")[1])
            if term not in currentQueryTermIndexDict:
                currentQueryTermIndexDict[termIndex] = term
        # print "currentQueryTermIndexDict:",currentQueryTermIndexDict
        
        for j in range(4,7):
            originalRawResultFileHandler.readline()            
        
        for j in range(0,K):
            currentLineElements = originalRawResultFileHandler.readline().strip().split(" ")
            if len(currentLineElements) == 65:
                base = 1
                for i in range(0,len(currentQueryTermIndexDict)):
                    term = str( currentQueryTermIndexDict[i] )
                    internal_doc_id = str( currentLineElements[63] )
                    external_trec_id = str( currentLineElements[64] )
                    totalBM25_score = float( currentLineElements[62] )
                    # newly updated for each training posting instance
                    partialBM25_score_component_part1 = float( currentLineElements[base + 10 + i] )
                    partialBM25_score_component_part2 = float( currentLineElements[base + 10 + 10 + i] )
                    partialBM25_score = float( currentLineElements[base + 10 + 10 + 10 + i] )
                    freq_in_collection = int( currentLineElements[base + 10 + 10 + 10 + i + 10] )
                    freq_in_doc = int( currentLineElements[base + 10 + 10 + 10 + i + 10 + 10] )
                    doc_words = int( currentLineElements[61] )
                    result_rank_for_this_posting = int( currentLineElements[0] )
                    
                    outputTrainingExample = str(result_rank_for_this_posting-1) + " " + currentQIDInStrFormat + " " + external_trec_id + " " + internal_doc_id + " " + term + " " + str(partialBM25_score_component_part1) + " " + str(partialBM25_score_component_part2) + " " + str(partialBM25_score) + " " + str(freq_in_collection) + " " + str(freq_in_doc) + " " + str(doc_words) + " " + str(totalBM25_score) + " " + str(result_rank_for_this_posting)
                    # for debug
                    # print outputTrainingExample
                    outputFileHandler.write(outputTrainingExample + "\n")
            else:
                pass

inputFileHandler.close()
outputFileHandler.close()
originalRawResultFileHandler.close()

print "Program Overall Processing Statistics:"
print "inputTestingFileName:",inputTestingFileName
print "originalRawResultFileName:",originalRawResultFileName
print "inputAuxFileName:",inputAuxFileName
print "outputFileName:",outputFileName
print "program ends."